#!/usr/bin/env python
# encoding: utf-8
"""
#-------------------------------------------------------------------#
#                   CONFIDENTIAL --- CUSTOM STUDIOS                 #     
#-------------------------------------------------------------------#
#                                                                   #
#                   @Project Name : Globallawonline                #
#                                                                   #
#                   @File Name    : wipo_treaty.py                      #
#                                                                   #
#                   @Programmer   : 李建                            #
#                                                                   #  
#                   @Start Date   : 2021/3/3 16:06                 #
#                                                                   #
#                   @Last Update  : 2021/3/3 16:06                 #
#                                                                   #
#-------------------------------------------------------------------#
# Classes:用于采集WIPO组织的东盟十国国际条约数据                                                          #
#                                                                   #
#-------------------------------------------------------------------#
"""
import hashlib
import re

import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

from ..items import MyFileItem


class Spiders(scrapy.Spider):
    name = 'wipo_treaty'
    allowed_domains = 'https://wipolex.wipo.int'
    # wipo国际条约网址
    start_urls = ['https://wipolex.wipo.int/en/treaties/members']
    # 目标国家名称
    Country = ["Brunei Darussalam", "Cambodia", "Indonesia", "Lao People's Democratic Republic", "Malaysia",
               "Myanmar", "Philippines", "Singapore", "Thailand", "Viet Nam"]
    SortAdict = {"Brunei Darussalam": "LAWCOUNTRYWL", "Cambodia": "LAWCOUNTRYJPZ", "Indonesia": "LAWCOUNTRYYDNXY",
                 "Lao People's Democratic Republic": "LAWCOUNTRYLW", "Malaysia": "LAWCOUNTRYMLXY",
                 "Myanmar": "LAWCOUNTRYMD", "Philippines": "LAWCOUNTRYFLB", "Singapore": "LAWCOUNTRYXJP",
                 "Thailand": "LAWCOUNTRYTG", "Viet Nam": "LAWCOUNTRYYN", }
    id = 0
    Current_page = ''

    def __init__(self, **kwargs):
        super(Spiders, self).__init__(**kwargs)
        # 使浏览器插件不显示
        # chrome_options = Options()
        # chrome_options.add_argument('--no-sandbox')
        # chrome_options.add_argument('--disable-dev-shm-usage')
        # chrome_options.add_argument('--headless')
        # self.driver = webdriver.Chrome(chrome_options=chrome_options)
        self.driver = webdriver.Chrome()  # 调用本地的谷歌浏览器

    def close(spider, reason):
        spider.driver.quit()    # 关闭浏览器

    def parse(self, response):
        """
        wipo国家按国家分类首页
        :param response: 一个网页返回类
        :return:
        """
        ctrs = response.xpath('//*[@id="wipo-int"]/wipolex-root/div/div/div/wipolex-member-parties/div/div[3]/div/div/div/div/div')     # 获取所有国家列表
        for ctr in ctrs:
            # 获取国家名称
            countryS = ctr.xpath('.//a/text()')
            country = ''
            if len(countryS):
                country = countryS.get().strip()
            # 国家名称在目标国家名称中则进入国家详情页
            if country in self.Country:
                countryUrlS = ctr.xpath('.//a')
                if len(countryUrlS):
                    yield response.follow(countryUrlS[0], callback=self.countryparse, dont_filter=True,
                                          meta={'country': country})

    def countryparse(self, response):
        """
        一个国家按签署的条约分类页面
        :param response: 一个网页返回类
        :return:
        """
        country = response.meta['country']
        trs = response.xpath('//wipolex-treaty-list/mat-table/mat-row')     # 国家签订的条约列表
        for tr in trs:
            # 国际条约标题
            titleS = tr.xpath('./mat-cell[1]//a/text()')
            title = ''
            if len(titleS):
                for titleSl in titleS:
                    title = title + ' ' + titleSl.get().strip()
                title = re.sub(r'''^ *''', r'''''', title)  # 去除开头的空格
            # 主办机构
            organiserS = tr.xpath('./mat-cell[2]//a/text()')
            organiser = ''
            if len(organiserS):
                organiser = organiserS.get().strip()
            # 签订日期
            publishDateS = tr.xpath('./mat-cell[3]/text()')
            publishDate = ''
            if len(publishDateS):
                for publishDateSl in publishDateS:
                    publishDate = publishDate + ' ' + publishDateSl.get().strip()
                publishDate = re.sub(r'''^ *''', r'''''', publishDate)  # 去除开头的空格
            # 生效日期
            effectiveDateS = tr.xpath('./mat-cell[4]/text()')
            effectiveDate = ''
            if len(effectiveDateS):
                for effectiveDateSl in effectiveDateS:
                    effectiveDate = effectiveDate + ' ' + effectiveDateSl.get().strip()
                effectiveDate = re.sub(r'''^ *''', r'''''', effectiveDate)  # 去除开头的空格
            # 国际条约详细地址
            detailUrlS = tr.xpath('./mat-cell[1]//a/@href')
            detailUrla = tr.xpath('./mat-cell[1]//a')
            detailUrl = ''
            if len(detailUrlS):
                detailUrl = self.allowed_domains[1] + detailUrlS.get().strip()
                if "WIPO" in organiser:
                    pass
                    # yield response.follow(detailUrla[0], callback=self.wipodetailparse, dont_filter=True,
                    #                       meta={'title': title, 'publishDate': publishDate,
                    #                             'effectiveDate': effectiveDate})
                else:
                    yield response.follow(detailUrla[0], callback=self.nwipodetailparse, dont_filter=True,
                                          meta={'country': country, 'title': title, 'publishDate': publishDate,
                                                'effectiveDate': effectiveDate})

    def wipodetailparse(self, response):
        """
        wipo主办的国际条约签约国等信息页
        :param response:
        :return:
        """
        country = response.meta['country']
        title = response.meta['title']
        publishDate = response.meta['publishDate']
        effectiveDate = response.meta['effectiveDate']
        trs = response.xpath(
            '//wipolex-accordion-row-treatytext//div[@class="mat-expansion-panel-body"]/div/div/div/div[@class="table-full"]/div')
        pass

    def nwipodetailparse(self, response):
        """
        不是wipo主办的国际条约签约国等信息页
        :param response:
        :return:
        """
        country = response.meta['country']
        title = response.meta['title']
        publishDate = response.meta['publishDate']
        effectiveDate = response.meta['effectiveDate']
        trs = response.xpath('//wipolex-accordion-row-treatytext//div[@class="mat-expansion-panel-body"]/div/div/div/div[@class="table-full"]/div')
        for tr in trs:
            lableS = tr.xpath('./div[1]/text()')
            lable = ''
            if len(lableS):
                lable = lableS.get().strip()
            if lable == "English":
                detaiUrlS = tr.xpath('./div[2]/a[1]/@href')
                detaiUrl = ''
                if len(detaiUrlS):
                    detaiUrl = self.allowed_domains[1] + detaiUrlS.get().strip()
                break
        statePartyaS = response.xpath('//wipolex-treaty-detail//div[@class="table-full"]/div[2]/div[2]/div[1]/a')
        if len(statePartyaS):
            statePartya = statePartyaS[a]
            yield response.follow(statePartya, callback=self.nwipodetailed, dont_filter=True,
                                  meta={'country': country, 'title': title, 'publishDate': publishDate,
                                        'effectiveDate': effectiveDate, 'detaiUrl': detaiUrl})

    def nwipodetailed(self, response):
        country = response.meta['country']
        title = response.meta['title']
        publishDate = response.meta['publishDate']
        effectiveDate = response.meta['effectiveDate']
        detaiUrl = response.meta['detaiUrl']
        # 获取缔约国
        statePartyS = response.xpath('//wipolex-parties/div/div/mat-table/mat-row/mat-cell[1]')
        statePartylist = []
        if len(statePartyS):
            for statePartySl in statePartyS:
                statePartySlr = statePartySl.get().strip()
                statePartylist.append(statePartySlr)
        stateParty = ",".join(statePartylist)
        yield response.follow(detaiUrl, callback=self.detailed, dont_filter=True,
                              meta={'country': country, 'title': title, 'publishDate': publishDate,
                                    'effectiveDate': effectiveDate, 'stateParty': stateParty})

    def detailed(self, response):
        item = MyFileItem()
        if response.status == 200:
            # 国别及文件信息
            country = response.meta['country']
            website = 'wipo'
            modular = 'regional'

            title = response.meta['title']
            publishDate = response.meta['publishDate']
            effectiveDate = response.meta['effectiveDate']
            stateParty = response.meta['stateParty']
            sortA = self.SortAdict[country]
            # 国际条约原文网址
            detailUrlS = response.xpath('//wipolex-text//div[@class="textFrame"]/iframe/@src')
            detailUrl = ''
            if len(detailUrlS):
                detailUrl = detailUrlS.get().strip()
            # 下载文件名
            fina = ''
            if len(detailUrl) > 0:
                fina = 'f' + str(hashlib.md5(detailUrl.encode('utf-8')).hexdigest())

            # 唯一ID
            systemid = str(hashlib.md5(detailUrl.encode('utf-8')).hexdigest())

            item['file_urls'] = ''
            item['country'] = country
            item['website'] = website
            item['modular'] = modular
            item['ext'] = 'pdf'
            item['fina'] = fina
            item['title'] = title
            item['detail'] = ''
            item['detailUrl'] = ''
            item['downloadUrl'] = detailUrl

            item['Title'] = title
            item['SortA'] = sortA
            item['StateParty'] = stateParty
            item['PublishDate'] = publishDate
            item['EffectiveDate'] = effectiveDate
            item['FullText'] = ''
            item['SortB'] = 'LANGUAGEYY'
            item['SortC'] = ''
            item['Articles'] = ''
            item['Chapter'] = ''
            item['Section'] = ''
            item['SYS_FLD_DIGITFILENAME'] = fina
            item['FileUrl'] = ''
            item['DownLoadUrl'] = detailUrl
            item['DownLoadWebNameC'] = '世界知识产权组织数据库'
            item['DownLoadWebNameE'] = "WIPO Lex"
            item['SYSID'] = systemid
            item['Website'] = 'Regional'
            item['Isconversion'] = '0'

            yield item

